#!/usr/bin/perl

use strict;
use warnings;
use Getopt::Long;
use WWW::Mechanize;
use Date::Calc qw (Today Day_of_Week Add_Delta_Days Date_to_Days);

my @array = ();

my $exchangeName="NYSE";
my $defaultPath = "/storage/Market/data";
my $operation;
my $stockName = "ADC";
my $result = GetOptions ("exchangeName=s" => \$exchangeName,
  					     "defaultPath=s"  => \$defaultPath,
                         "operation=s"    => \$operation,
                         "stockName=s"    => \$stockName
                         );

my $url="http://eoddata.com";
my $mech = WWW::Mechanize->new( autocheck => 1 );

#getExchangeData("$exchangeName","$stockName"); die;
#isUpToDate("$exchangeName","$stockName");
#updateExchangeData("$exchangeName"); die;
#updateStockData("$exchangeName","$stockName"); die;

if (defined($operation)) {
	
	if ($operation eq "getExchange") {
		print "Performing full $exchangeName download!\n";
		unless (getExchangeData("$exchangeName") != 0) {
			print "Update completed\n";
		} else {
			print "$exchangeName download failed!\n";
			return 1;
		}
		
	} elsif ($operation eq "updateStock") {
		print "Performing $stockName update! \n";
		unless (updateStockData("$exchangeName","$stockName") != 0) {
			print "updateStockData stock data completed!\n";
		} else {
			print "updateStockData stock data failed!\n";
			return 1;
		}
		
	} elsif ($operation eq "getExchangesList") {
		print "Performing getExchangesList update! \n";
		unless (getExchangesList() != 0) {
			print "getExchangesList completed!\n";
		} else {
			print "getExchangesList failed!\n";
			return 1;
		}
	
	} elsif ($operation eq "getSymbolsList") {
		print "Performing getSymbolsList update! \n";
		unless (getSymbolsList($exchangeName) != 0) {
			print "getSymbolsList completed!\n";
		} else {
			print "getSymbolsList failed!\n";
			return 1;
		}
	
	} elsif ($operation eq "list") {
		print "getExchange\n";
		print "updateStock\n";
		print "getExchangesList\n";
		print "getSymbolsList\n";
	} else {
		die "\"$operation\" operation unknown!\n";
	} 
	
} else {
	(die "No or unknown operation provided!!\n");
} 

sub logIn {
	my $subName = (caller(0))[3];
	print "Entered $subName...\n";

	my $login=login;
	my $password=password;

	$mech->add_header(
	"Connection" => "keep-alive",
    "Keep-Alive" => "115");

	$mech->get($url);
	if ($mech->success()) {
		print "Fetched $url page\n";
	} else {
		die ("Error fetching the $url\n");
	}

	$mech->submit_form(
    	form_name => 'aspnetForm',
    	fields => {
        	'ctl00$cph1$lg1$txtEmail' => $login,
        	'ctl00$cph1$lg1$txtPassword' => $password,
    	},
    	button => 'ctl00$cph1$lg1$btnLogin',
	);

	my $mainPageContent = $mech->content();

	## Login Validation 
	if ( $mainPageContent =~ /Invalid/ ) {
		print "Invalid Username or Password when logging to $url\n";
		print "Exiting $subName...\n";
		return 1;
	} elsif ($mainPageContent =~ /Evgeny/) {
		print "Logged in $url successfully\n";
		print "Exiting $subName...\n";
		return 0;
	} else {
		print "Error loading the page\n";
		print "Exiting $subName...\n";
		return 2;
	}
}

sub getExchangesList {
my $subName = (caller(0))[3];
print "Entered $subName...\n";
my $data;

foreach my $line (split qr/\R/, $mech->content) {
if ($line=~ m/t\/([A-Z]+)/){
	$data .= "$1\n";
}
}
writeToFile("exchanges_list.txt","exchanges",$data);
print "Exiting $subName...\n";
return 0;
}
 
sub writeToFile {
	#use File::Path;
	use File::Checksum;
  
	my $subName = (caller(0))[3];
	print "Entered $subName...\n";
	my ($fileName, $relativePath, $data, $flag) = @_;
    
    unless (defined($flag)){ #This is just to keep NAY value in $flag. It should be done in a smarter way
    	$flag = "NULL";
    }
    
    # Path Validation
    validatePath("$relativePath");
    
    unless ($flag eq "a") {
    	print "Performing create/overwrite\n";
      # File Validation
    unless (-e "$defaultPath/$relativePath/$fileName") {
    	print "$defaultPath/$relativePath/$fileName does not exists. Creating...\n";
    	open FileHandle, "> $defaultPath/$relativePath/$fileName" or die("$!");
		print FileHandle $data;
        close FileHandle;
    } else {
    	print "$defaultPath/$relativePath/$fileName already exists. Comparing...\n";
    	open FileHandle, "> $defaultPath/$relativePath/$fileName.tmp" or die("$!");
    	print FileHandle $data;
    	close FileHandle;
   
    	my $checksumNewFile = Checksum("$defaultPath/$relativePath/$fileName.tmp", 100);
    	my $checksumCurrentFile = Checksum("$defaultPath/$relativePath/$fileName", 100);
   
    	unlink("$defaultPath/$relativePath/$fileName.tmp") or die("$!");
       	
    if ($checksumCurrentFile == $checksumNewFile) {
    	print "The checksums are similar, skipping write...\n";
    return;
    } else {
    	print "The checksums are not equal, updating...\n";
    	open FileHandle, "> $defaultPath/$relativePath/$fileName" or die("$!");
        print FileHandle $data;
        close FileHandle;
      }
    }
    } elsif ($flag eq "a") {
    	print"Performing append\n";
    	
    	open my $in,  '<',  "$defaultPath/$relativePath/$fileName"     or die "Can't read old file: $!";
		open my $out, '>', "$defaultPath/$relativePath/$fileName.tmp" or die "Can't write new file: $!";

		print $out "$data"; #Appending the new lines at the beginning of the file

		while( <$in> ) {
			chomp; chomp;        # Eat up to two newlines, just in case
			next if /^\s*$/;     # Line is empty or contains whitespace
			print $out "$_\n";   # Add all the rest of the lines which were already existing in $fileName
		}
		
		close $out;
		
		rename("$defaultPath/$relativePath/$fileName.tmp", "$defaultPath/$relativePath/$fileName") || die ( "Error in renaming" );
		unlink("$defaultPath/$relativePath/$fileName.tmp");
		
    }
    print "Exiting $subName...\n";
    return 0;
}

sub validatePath {
	use File::Path;
	my $subName = (caller(0))[3];

	print "Entered $subName...\n";
	my ($relativePath) = @_;
	unless (-d "$defaultPath/$relativePath"){
	    print "The $defaultPath/$relativePath directory does not exist, creating...\n";
	    mkpath ("$defaultPath/$relativePath") or die("$!");
	    }
	
	print "Exiting $subName...\n";
}
 
sub getSymbolsList {
	my $subName = (caller(0))[3];
	print "Entered $subName...\n";
	my $data;
	my ($exchangeName) = @_;
    
    unless (-e "$defaultPath/exchanges/$exchangeName.txt") {
    	print "The symbols list for $exchangeName does not exist. Going to download.\n";
    	
    	unless (logIn != 0) {
    		$mech->get("$url/Data/symbollist.aspx?e=$exchangeName");
	    	validatePath("exchanges/");
              
    		foreach my $line (split qr/\R/, $mech->content) {
    		$data .= "$line\n" unless ($line  =~ m/Symbol/g);
    	}
       	writeToFile("$exchangeName.txt","exchanges",$data);
       	
   		} else {
    		print "Seems that login was not successful, exiting $subName...\n";
    	}
    			
    } else {
    	print "The symbols list for $exchangeName already exists.\n";
    }
    
    print "Exiting $subName...\n\n";
    return 0;
}

sub getStockData {
	my $subName = (caller(0))[3];
	print "Entered $subName...\n";
	
	my($day, $tmonth, $tyear)=(localtime)[3,4,5];
	my $month = $tmonth+1;
	my $year = $tyear+1900;
	#print"$day, $month, $year\n";
   	
	my ($exchangeName,$stockName) = @_;
	
	#Check if the file already exists
	unless (-e "$defaultPath/exchanges/$exchangeName/$stockName.txt") {
		print "$stockName.txt does not exist yet, going to create\n";
		
		my $data;
		my $yahoo = WWW::Mechanize->new(autocheck => 1);
		eval {$yahoo->get("http://ichart.finance.yahoo.com/table.csv?s=$stockName&amp;d=$day&amp;e=$month&amp;f=\$&amp;g=d&amp;a=7&amp;b=19&amp;c=1950&amp;ignore=.csv");};
		if ($@) {
			print "$stockName symbol does not exist on Yahoo! This stock file will not be created.\n";
			print "Exiting $subName...\n\n\n";
			return;
		}
		
			foreach my $line (split qr/\R/, $yahoo->content) {
			$data .= "$line\n" unless ($line  =~ m/Open/g);
		}
		
	    writeToFile("$stockName.txt","exchanges/$exchangeName",$data) && print"$stockName.txt saved!\n";
	    print "Exiting $subName...\n";
	} else {
		print "$stockName.txt already exists, going to update\n";
		updateStockData("$exchangeName","$stockName");
	}
	print "Exiting $subName...\n\n\n";	
}

sub getExchangeData {
	my $subName = (caller(0))[3];
	print "Entered $subName...\n";

	my ($exchangeName) = @_;
	print "Going to get full exchange data for $exchangeName\n";
	getSymbolsList("$exchangeName");
	
	open my $readFileHandle, "< $defaultPath/exchanges/$exchangeName.txt" or die("$!");
 
	while (<$readFileHandle>) {
		$_ =~ m/(^([A-Z]+)([-.][A-Z])?)/g;
		print "Going to get EOD data for $1\n";
		getStockData("$exchangeName","$1");
	}
             
    close $readFileHandle;
    print "Exiting $subName...\n\n\n";
    return 0;
}    	

sub isUpToDate {
	my $subName = (caller(0))[3];
	print "Entered $subName...\n";
		
	my ($exchangeName,$stockName) = @_;
	my ($day, $tmonth, $tyear)=(localtime)[3,4,5];
	my $month = $tmonth+1;
	my $year = $tyear+1900;
	my @lastTradingDate; #The last trading day date  
	my @lastStockDate; #The last date date recorded for a stock
	my $dowName;
	
	my @today = Today; #Today's date
	my $dow = Day_of_Week($year,$month,$day); #Day of week
	
	if ($dow == 1) {
		$dowName = "Monday";
	} elsif ($dow == 2) {
		$dowName = "Tuesday";
	} elsif ($dow == 3) {
		$dowName = "Wednesday";
	} elsif ($dow == 4) {
		$dowName = "Thursday";
	} elsif ($dow == 5) {
		$dowName = "Friday";
	} elsif ($dow == 6) {
		$dowName = "Saturday";
	} elsif ($dow == 2) {
		$dowName = "Sunday";
	}
	 
	print "Todays is @today and this is $dowName\n";
	
	
	if ($dow == 1) {
		@lastTradingDate=Add_Delta_Days(@today, -3);
		print "Last available trading date should be @lastTradingDate\n";
	} elsif  ($dow == 7) {
		@lastTradingDate=Add_Delta_Days(@today, -2);
		print "Last available trading date should be @lastTradingDate\n";
	} else {
		@lastTradingDate=Add_Delta_Days(@today, -1);
		print "Last available trading date should be @lastTradingDate\n";
	}
	
	($lastStockDate[0],$lastStockDate[1],$lastStockDate[2]) = getLastStockDate("$exchangeName","$stockName");
	print "Last available date record for $stockName is @lastStockDate\n";

	if (Date_to_Days("$lastTradingDate[0]","$lastTradingDate[1]","$lastTradingDate[2]") != Date_to_Days("$lastStockDate[0]","$lastStockDate[1]","$lastStockDate[2]")) {
		print "The $stockName data is outdated\n";
		return @lastStockDate;
	} else {
		print "The $stockName data is up to date\n";
		return @lastStockDate = ();
	}
	#die; #Cheking, do not delete yet. 
	print "Exiting $subName...\n";
}

sub getLastStockDate {
	# Determine what is the last date in a stock file. Just reads the first line of the stock file and gets the date acceptible for isUpToDate sub
	my $subName = (caller(0))[3];
	print "Entered $subName...\n";
	
	my ($exchangeName,$stockName) = @_;
	my @lastAvailableDate;
	
	
	open my $file, '<', "$defaultPath/exchanges/$exchangeName/$stockName.txt";
	my $firstLine = <$file>; 
	close $file;
	#print "$firstLine\n";
	
	my @columns = split (',', $firstLine);
	($lastAvailableDate[0],$lastAvailableDate[1],$lastAvailableDate[2]) = split('-', $columns[0]); 
	
	print "Exiting $subName...\n";	
	return @lastAvailableDate;
}

sub updateStockData {
	my $subName = (caller(0))[3];
	print "Entered $subName...\n";
	
	my ($exchangeName,$stockName) = @_;
	my $relativePath = "exchanges/$exchangeName";
	
	unless (-e "$defaultPath/$relativePath/$stockName.txt") {
		print "$stockName does not exist, skipping to the next one...\n";
		print "Exiting $subName...\n";
		return 1;
	}
	
	my @today = Today; #Today's date
	my ($endYear, $endMonth, $endDay) = ($today[0], $today[1], $today[2]);
	
	my @result = isUpToDate("$exchangeName","$stockName");
	if (@result) {
		print "Going to update $stockName \n"
	} else {
		print "No update needed. Exiting... $subName\n";
		print "Exiting $subName...\n";
		return 0;
	}
	
	my @newStartDate = Add_Delta_Days(@result, +1); #Date when the update will start.
	print "The $stockName will be updated starting from @newStartDate\n";
	
	my ($startYear, $startMonth, $startDay) = ($newStartDate[0],$newStartDate[1],$newStartDate[2]);
	my $modifiedStartMonth = $startMonth-1; # Modified start and end months is used to match Yahoo format. It's using "month-1" format. 
	my $modifiedEndMonth = $endMonth-1;
	
	print "Going to generate Yahoo string using year $startYear, month $startMonth and day $startDay\n";
	print "http://ichart.finance.yahoo.com/table.csv?s=$stockName&a=$modifiedStartMonth&b=$startDay&c=$startYear&d=$modifiedEndMonth&e=$endDay&f=$endYear&g=d&ignore=.csv\n";
	     
	my $data;
	my $yahoo = WWW::Mechanize->new(autocheck => 1);
	
	eval {$yahoo->get("http://ichart.finance.yahoo.com/table.csv?s=$stockName&a=$modifiedStartMonth&b=$startDay&c=$startYear&d=$modifiedEndMonth&e=$endDay&f=$endYear&g=d&ignore=.csv");};
	if ($@) {
		print"$stockName does not exist!"
	}
	
	print $yahoo->content();
	
	foreach my $line (split qr/\R/, $yahoo->content) {
		$data .= "$line\n" unless ($line  =~ m/Open/g);
	}
	
	#print "$data\n"; die;
	
	validatePath("exchanges/$exchangeName");
    writeToFile("$stockName.txt","exchanges/$exchangeName",$data, "a") && print"$stockName.txt saved!\n";
    print "Exiting $subName...\n";
    return 0;
} 
 
#sub updateExchangeData {
#	my $subName = (caller(0))[3];
#	print "Entered $subName...\n";
#	
#	my ($exchangeName) = @_;
#	getSymbolsList("$exchangeName");
#	
#	open my $readFileHandle, "< $defaultPath/exchanges/$exchangeName.txt" or die("$!");
# 
#	while (<$readFileHandle>) {
#		$_ =~ m/(^([A-Z]+)([-.][A-Z])?)/g;
#		print "$1\n";
#		updateStockData("$exchangeName","$1");
#	}
#             
#    close $readFileHandle;
#	print "Exiting $subName...\n";
#}